# -*- coding: utf-8 -*-
# @Time    : 2021/2/20 13:27
# @Author  : AkromA
# @File    : ajk_zf_crawler.py
# @Software: PyCharm
# @Info    : 安居客租房数据爬虫（页面格式较规整暂用正则不用bs4实现）

import re
import time
import random
import requests
import traceback
from urllib.request import quote
from db.db_conn import DBHandle
from oss.oss_upload_img import OssUpload

headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
    "Accept-Language": "zh-CN,zh;q=0.9",
    "Accept-Encoding": "gzip, deflate, br",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.100 Safari/537.36",
}


# 转换特色
def features_handle(param):
    features_list = []
    f_list = param.split(",")
    for f in f_list:
        if "近地铁" in f: features_list.append("48")
        elif "近学校" in f: features_list.append("49")
        elif "交通便利" in f: features_list.append("50")
        elif "近商业区" in f: features_list.append("51")
        elif "近医院" in f: features_list.append("108")
        elif "近公交站台" in f: features_list.append("112")
        elif "近商超" in f: features_list.append("113")
    if len(features_list) > 1:
        return ",".join(features_list)
    elif len(features_list) == 1:
        return features_list[0]
    else:
        return "0"


# 卧室设施处理
def bedroom_equipment_handle(param):
    bedroom_equipment_list = []
    if "电视" in param:bedroom_equipment_list.append("42")
    if "空调" in param:bedroom_equipment_list.append("43")
    if "衣柜" in param:bedroom_equipment_list.append("44")
    if "床" in param:bedroom_equipment_list.append("45")
    if "卫生间" in param:bedroom_equipment_list.append("46")
    if "智能门锁" in param:bedroom_equipment_list.append("47")
    if "阳台" in param:bedroom_equipment_list.append("109")
    if "暖气" in param:bedroom_equipment_list.append("110")
    if len(bedroom_equipment_list) == 1:
        return bedroom_equipment_list[0]
    elif len(bedroom_equipment_list) > 1:
        return ",".join(bedroom_equipment_list)
    else:
        return ""


# 客厅设施处理
def communal_facilities_handle(param):
    communal_facilities_list = []
    if "冰箱" in param: communal_facilities_list.append("147")
    if "洗衣机" in param: communal_facilities_list.append("148")
    if "热水器" in param: communal_facilities_list.append("149")
    if "宽带" in param: communal_facilities_list.append("150")
    if "沙发" in param: communal_facilities_list.append("151")
    if "油烟机" in param: communal_facilities_list.append("152")
    if "燃气灶" in param: communal_facilities_list.append("153")
    if "可做饭" in param: communal_facilities_list.append("154")
    if len(communal_facilities_list) == 1:
        return communal_facilities_list[0]
    elif len(communal_facilities_list) > 1:
        return ",".join(communal_facilities_list)
    else:
        return ""


# 出租要求处理
def requirements_handle(param):
    requirements_list = []
    if "只限女生" in param:
        requirements_list.append("205")
    if "一家人" in param:
        requirements_list.append("206")
    if "禁止养宠物" in param:
        requirements_list.append("207")
    if "半年起租" in param:
        requirements_list.append("208")
    if "一年起租" in param:
        requirements_list.append("209")
    if "租户稳定" in param:
        requirements_list.append("210")
    if "作息正常" in param:
        requirements_list.append("211")
    if "禁烟" in param:
        requirements_list.append("212")
    if "禁止吸烟" in param:
        requirements_list.append("212")

    if len(requirements_list) == 1:
        return requirements_list[0]
    elif len(requirements_list) > 1:
        return ",".join(requirements_list)
    else:
        return ""


# 押付处理
def payment_handle(param):
    if "付3" in param:
        return 64
    elif "付1" in param:
        return 63
    elif "半年付" in param:
        return 65
    elif "年付" in param:
        return 66
    elif "面议" in param:
        return 129
    else:
        return 0


# 朝向
def cx_handle(param):
    if param == "东": return 55
    elif param == "南": return 56
    elif param == "西": return 57
    elif param == "北": return 58
    elif param == "东南": return 59
    elif param == "东北": return 60
    elif param == "西南": return 61
    elif param == "西北": return 62
    elif param == "南北": return 144
    elif param == "东西": return 145
    else: return 144      # 默认值为南北朝向


# 插入数据并获取id
def insert_data(sql):
    db_handle = DBHandle()
    data_id = db_handle.insert_one(sql)
    db_handle.close_db()
    return data_id


# 查找数据
def select_data(sql):
    db_handle = DBHandle()
    flag = db_handle.select_sql(sql)
    db_handle.close_db()
    return flag


# 转换建筑类型
def house_type_handle(param):
    if "普通住宅" in param: return 132
    elif "公寓" in param: return 133
    elif "别墅" in param: return 134
    elif "平房" in param: return 169
    elif "四合院" in param: return 170
    else:
        return 136


# 转换装修类型
def decoration_handle(param):
    if "精装修" in param: return 52
    elif "简单装修" in param: return 53
    elif "豪华装修" in param: return 54
    elif "毛坯" in param: return 146
    else:
        return 0


# 获取图片阿里云的地址
def get_img_url(img_url):
    # 初始化阿里云图片上传接口
    oss_upload = OssUpload()
    # 处理图片地址re
    pattern = r'(.*?)\.\d+x\d+'

    try:
        # 处理图片地址
        if len(re.findall(pattern, img_url)) > 0:
            img_link = re.findall(pattern, img_url)[0]
        elif "!" in img_url:
            img_link = img_url.split('!')[0]
        else:
            img_link = img_url

        bytes_con = requests.get(img_link).content
        bytes_name = img_link.split('/')[-2] + "_" + img_link.split('/')[-1]

        bytes_name = bytes_name.replace('?t=1&srotate=1', '')

        ali_img_url = oss_upload.upload_byte(bytes_name, bytes_con)
        return ali_img_url
    except Exception as error:
        return False


# 获取页面源代码html格式
def get_html(url_):
    # print(url_)
    response_ = requests.get(url_, headers=headers, allow_redirects=False)
    # 通过请求状态值判断是否请求成功
    # 需要判断是否被限制IP
    if response_.status_code != 200:
        log_info = "获取页面html时出错"
        print(log_info)
        return False

    return response_.text


# 通过搜索关键词、页数、城市缩写，返回搜索后页面详情页链接列表。链接中关键词可以为区名或商圈，l2表示个人房源，x1表示整租，px3表示按最新发布降序排列
def get_content_links(city_, rent_type, page_num, param, crawler_type):
    url_ = "https://%s.zu.anjuke.com/fangyuan/l2-x%s-px3-p%s/?t=1&comm_exist=on&kw=%s" % (city_, rent_type, page_num, quote(param))
    resp = requests.get(url_, headers=headers, allow_redirects=False)

    if resp.status_code != 200:
        print("连接有问题， %s" % resp.status_code)
        return

    html_= resp.text

    cut_pattern = r'<!--区域板块租房房源列表页-->([\s\S]*?)<!--零少结果推荐文案,置于最底部-->'
    content_list_con = re.findall(cut_pattern, html_)
    if not content_list_con:
        print("没有找到列表位置")
        return

    con_pattern = r'<div class="zu-itemmod" link="(.*?)"[\s\S]*?<img[\s\S]*?src="(.*?)"'
    link_list_ = re.findall(con_pattern, content_list_con[0])
    if not link_list_:
        print("没有找到链接列表位置")
        return

    # print(len(link_list_))

    for con_link in link_list_[7:10]:
        con_html = get_html(con_link[0])
        tu_url = con_link[1]

        # 20210302增加，从url中过去当前数据id，用来判断是否重复
        get_id_patt = r'[\s\S]*?/(\d+)\?is'
        house_ajk_id = re.findall(get_id_patt, con_link[0])[0]

        # 判断重复
        judge_sql = "SELECT id FROM 08_house WHERE fid = '%s' AND crawler_type = '%s' AND state = '1' AND type = '2';" % (house_ajk_id, crawler_type)
        flag = select_data(judge_sql)
        if len(flag) > 0:
            continue

        # 随机sleep，防止被封
        time.sleep(random.randrange(7, 10))
        yield con_html, con_link[0], tu_url, house_ajk_id


# 获取详情页内容
def con_handle(html_, city_code, r_type, list_img_url, ajk_h_id, xq_url, crawler_type):

    # 各字段获取开始
    # ################
    # ################
    # house表

    # 固定值
    type_ = "1"  # 区分房源 1租房 2二手房 3新房

    try:
        # 页面上获取的字段

        # 室厅
        rhval = ""
        patt_rhval = r'<span class="type">户型：</span>[\s\S]*?<span class="info">(.*?)</span>'
        con_rhval = re.findall(patt_rhval, html_)
        if con_rhval:
            rhval = re.sub(r'<[\s\S]*?>', '', con_rhval[0]).replace('室', ',').replace('厅', ',').replace('卫', '')
            # print(rhval)

        # 面积
        area = ""
        patt_area = r'<span class="type">面积：</span>[\s\S]*?<span class="info">(.*?)</span>'
        con_area = re.findall(patt_area, html_)
        if con_area:
            area = re.sub(r'<[\s\S]*?>', '', con_area[0]).replace('平方米', '').replace('平方米', '')
            # print(area)

        # 标签
        tags_list = []
        patt_tag = r'<ul class="title-label cf">([\s\S]*?)</ul>'
        patt_tag_con = r'<li[\s\S]*?>(.*?)</li>'
        con_tag = re.findall(patt_tag, html_)
        if con_tag:
            tags_list = re.findall(patt_tag_con, con_tag[0])
            # print(tags_)
        if len(tags_list) == 1:
            tags = tags_list[0]
        elif len(tags_list) > 1:
            tags = ",".join(tags_list)
        else:
            tags = ""

        # 朝向
        house_orient_name = ""
        patt_toward = r'<span class="type">朝向：</span>[\s\S]*?<span class="info">(.*?)</span>'
        con_toward = re.findall(patt_toward, html_)
        if con_toward:
            house_orient_name = re.sub(r'<[\s\S]*?>', '', con_toward[0]).replace('朝', '')
            # print(house_orient_name)

        # 长标题
        long_title = ""
        patt_title = r'<h3 class="house-title">[\s\S]*?<div[\s\S]*?>(.*?)</div>'
        con_title = re.findall(patt_title, html_)
        if con_title:
            long_title = re.sub(r'<[\s\S]*?>', '', con_title[0])
            # print(long_title)

        # 装修
        fitment_value = ""
        patt_fitment = r'<span class="type">装修：</span>[\s\S]*?<span class="info">(.*?)</span>'
        con_fitment = re.findall(patt_fitment, html_)
        if con_fitment:
            fitment_value = con_fitment[0]
            # print(fitment_value)

        # 房屋类型
        house_type_con = ""
        patt_house_type = r'<span class="type">类型：</span>[\s\S]*?<span class="info">(.*?)</span>'
        con_house_type = re.findall(patt_house_type, html_)
        if con_house_type:
            house_type_con = con_house_type[0]
            # print(house_type_con)

        # 区名
        region_name = ""
        # 商圈名
        block_name = ""
        # 小区名称
        comm_name = ""
        patt_comm = r'<span class="type">小区：[\s\S]*?<a href="[\s\S]*?>(.*?)</a>[\s\S]*?<a href="[\s\S]*?>(.*?)</a>[\s\S]*?<a href="[\s\S]*?>(.*?)</a>'
        con_comm = re.findall(patt_comm, html_)
        if con_comm:
            comm_name = con_comm[0][0]
            block_name = con_comm[0][2]
            region_name = con_comm[0][1]

            # print(region_name, block_name, comm_name)

        # 价格，押付
        price = ""
        rent_ = ""
        patt_price = r'<li class="full-line cf">[\s\S]*?<span[\s\S]*?>(.*?)</span>[\s\S]*?<span class="type">(.*?)</span>'
        con_price = re.findall(patt_price, html_)
        if con_price:
            price = re.sub(r'<[\s\S]*?>', '', con_price[0][0]).replace('元/月', '')
            # print(price)
            rent_ = con_price[0][1]
            # print(rent_)

        # 楼层（总数）
        floor = ""
        # 楼宇
        floor_tag = ""
        patt_floor = r'<span class="type">楼层：</span>[\s\S]*?<span class="info">(.*?)</span>'
        con_floor = re.findall(patt_floor, html_)
        if con_floor:
            floor_content = re.sub(r'<[\s\S]*?>', '', con_floor[0])
            if "(" in floor_content:
                floor_tag = floor_content.split("(")[0]
            else:
                floor_tag = ""
            patt_floor_a = r'共(\d+)层'
            floor_ = re.findall(patt_floor_a, floor_content)
            if floor_: floor = floor_[0]

        # 发布时间
        # time_ = ""
        # patt_pub_time = r'<span id="houseCode">[\s\S]*?发布时间：<b[\s\S]*?>(.*?)</b>'
        # con_pub_time = re.findall(patt_pub_time, html_)
        # if con_pub_time:
        #     pub_time = con_pub_time[0]
        #     timeArray = time.strptime(pub_time, "%Y年%m月%d日")
        #     # print(pub_time)
        #     # 转为时间戳
        #     time_ = int(time.mktime(timeArray))
        #     # print(time_)

        # 20210302发布时间修改为入库时间
        time_ = int(time.time())

        # 描述
        description = ""
        patt_xq = r'<div class="houseInfo-item-desc[\s\S]*?<b class="strongbox[\s\S]*?>([\s\S]*?)</b>'
        patt_xq_a = r'<div class="auto-general">[\s\S]*?<b class="strongbox"[\s\S]*?>([\s\S]*?)</b>'
        con_xq = re.findall(patt_xq, html_)
        con_xq_a = re.findall(patt_xq_a, html_)
        if con_xq:
            description = re.sub(r'<[\s\S]*?>', '', con_xq[0])
            # print(description)
        elif con_xq_a:
            description = re.sub(r'<[\s\S]*?>', '', con_xq_a[0])
            # print(description)

        # 室内图
        room_photos_list = []
        patt_snt = r'<!--室内图-->([\s\S]*?)<!--户型图-->'
        patt_snt_list = r'<img data-src="(.*?)"'
        con_snt = re.findall(patt_snt, html_)
        if con_snt:
            con_snt_cut = con_snt[0]
            con_snt_list = re.findall(patt_snt_list, con_snt_cut)
            if con_snt_list:
                # print(con_snt_list)
                for i in con_snt_list:
                    room_photos_link = get_img_url(i)
                    if room_photos_link: room_photos_list.append(room_photos_link)
        if len(room_photos_list) == 1:
            room_photos = room_photos_list[0]
        elif len(room_photos_list) > 1:
            room_photos = ",".join(room_photos_list)
        else:
            room_photos = ""

        # 户型图
        model_photos = ""
        patt_hxt = r'<!--户型图-->([\s\S]*?)<!--环境图-->'
        patt_hxt_list = r'<img data-src="(.*?)"'
        con_hxt = re.findall(patt_hxt, html_)
        if con_hxt:
            con_hxt_cut = con_hxt[0]
            con_hxt_list = re.findall(patt_hxt_list, con_hxt_cut)
            if con_hxt_list:
                model_photos_link = con_hxt_list[0]
                # print(model_photos_link)
                model_photos = get_img_url(model_photos_link)

            # 更新时间
        update_time = int(time.time())

        # 需要查id的字段

        # 朝向id
        towards = cx_handle(house_orient_name)

        # 房屋类型id
        house_type = ""
        if house_type_con: house_type = house_type_handle(house_type_con)

        # 装修id
        decoration = ""
        if fitment_value: decoration = decoration_handle(fitment_value)

        # fid(暂时存放安居客租房房屋id，用于排重)
        fid = ajk_h_id

        # 供暖方式id（页面上暂无）
        heating = ""

        # 特色标签id
        features = features_handle(tags)

        # 是否有电梯
        elevator = 0
        if "电梯" in tags: elevator = 1

    except Exception as error:
        print("获取house参数出错 %s" % error)
        traceback.print_exc()
        return

    #
    # 列表图片
    img = ""
    img_flag = get_img_url(list_img_url)
    if img_flag: img = img_flag

    # 需要去数据库中获取的字段（通过小区名称）
    # 区域id
    areaCode = ""
    # 小区id
    comm_id = ""
    # 商圈id
    block_id = ""
    # 经纬度
    lat = ""
    # 经纬度
    lng = ""
    if comm_name:
        sel_sql = "SELECT comm_id, areaCode, block_id, lat, lng FROM 06_comm WHERE state = 1 AND comm_name = \"%s\" AND areaCode LIKE \"%s%%\";" % (comm_name, city_code)
        flag = select_data(sel_sql)
        if len(flag) > 0:
            comm_id = flag[0]['comm_id']  # 小区id
            areaCode = flag[0]['areaCode']  # 区域id
            block_id = flag[0]['block_id']  # 商圈id
            lat = flag[0]['lat']  # 经纬度
            lng = flag[0]['lng']  # 经纬度
        else:
            # 可能需要新增该小区，暂时不添加这部分逻辑
            print("%s 小区目前还不存在，待添加" % comm_name)
            return
    else:
        print("页面上没有小区名称")
        return

    # 拼SQL
    insert_sql = "INSERT INTO 08_house (`areaCode`, `rhval`, `area`, `comm_id`, `tags`, `house_orient_name`, `block_id`, `type`, `long_title`, `region_name`, `price`, `img`, `floor_tag`, `state`, `block_name`, `floor`, `towards`, `time`, `description`, `room_photos`, `features`, `lat`, `lng`, `update_time`, `elevator`, `house_type`, `heating`, `fitment_value`, `decoration`, `model_photos`, `fid`, `crawler_type`) VALUES ("
    insert_sql += "\"%s\", \"%s\", \"%s\", \"%s\", " % (areaCode, rhval, area, comm_id)
    insert_sql += "\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", " % (tags, house_orient_name, block_id, type_, long_title)
    insert_sql += "\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", " % (region_name, price, img, floor_tag, '1')
    insert_sql += "\"%s\", \"%s\", \"%s\", \"%s\", " % (block_name, floor, towards, time_)
    insert_sql += "\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", " % (description, room_photos, features, lat, lng)
    insert_sql += "\"%s\", \"%s\", \"%s\", \"%s\", " % (update_time, elevator, house_type, heating)
    insert_sql += "\"%s\", \"%s\", \"%s\", \"%s\", \"%s\");" % (fitment_value, decoration, model_photos, fid, crawler_type)

    house_id = insert_data(insert_sql)
    print(house_id)

    # ################
    # ################
    # rent_info表

    # 所属房源id
    hid = house_id
    # 整租合租，1为整租2为合租
    rent_model = r_type
    # 押X付Y
    payment = payment_handle(rent_)
    # 租金
    rent_price = price
    # 面积
    rental_area = area
    # 卧室朝向
    bedroom_towards = towards

    try:
        # 房间设施
        has_list = ""
        patt_has = r'<li class="peitao-item has">[\s\S]*?<div class="peitao-info">(.*?)</div>'
        con_has = re.findall(patt_has, html_)
        if con_has:
            has_list = con_has
            # print(has_list)
        # 卧室设施
        bedroom_equipment = bedroom_equipment_handle(has_list)
        # 客厅设施
        communal_facilities = communal_facilities_handle(has_list)

        # 出租房间类型（主次卧）
        bedroom_type = 200
        patt_bedroom_type = r'<span class="type">房间：</span>[\s\S]*?<span class="info">(.*?)</span>'
        con_bedroom_type = re.findall(patt_bedroom_type, html_)
        if con_bedroom_type:
            bedroom_type_con = con_bedroom_type[0]
            if "主卧" in bedroom_type_con:
                bedroom_type = 198
            elif "次卧" in bedroom_type_con:
                bedroom_type = 199
            else:
                bedroom_type = 200

        # 看房时间（页面上无数据）
        open_home = 0

        # 出租要求
        requirements = ""
        patt_requirements = r'<div class="houseInfo-item-desc">[\s\S]*?<b class="strongbox"[\s\S]*?>([\s\S]*?)</b>'
        con_requirements = re.findall(patt_requirements, html_)
        if con_requirements:
            requirements = re.sub(r'<[\s\S]*?>', '', con_requirements[0])
            # print(requirements)
        rental_requirements = requirements_handle(requirements)

    except Exception as error:
        print("获取rent_info参数出错 %s" % error)
        traceback.print_exc()
        return

    insert_info_sql = "INSERT INTO 08_rent_info (`hid`, `rent_model`, `payment`, `bedroom_equipment`, `communal_facilities`, `rent_price`, `rental_area`, `bedroom_type`, `bedroom_towards`, `open_home`, `rental_requirements`, `url`) VALUES ("
    insert_info_sql += "\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", " % (hid, rent_model, payment, bedroom_equipment, communal_facilities)
    insert_info_sql += "\"%s\", \"%s\", \"%s\", \"%s\", " % (rent_price, rental_area, bedroom_type, bedroom_towards)
    insert_info_sql += "\"%s\", \"%s\", \"%s\");" % (open_home, rental_requirements, xq_url)

    house_info_id = insert_data(insert_info_sql)
    print(house_info_id)
    # 各字段获取结束
    # ################
    # ################


# 主方法
def ajk_zf_main(city_short, district_param, city_code, data_source, rent_type):

    # 传入参数为city_short城市缩写，city_code为该城市代码的前4位，district_param商圈或区或其他关键词，data_source目标网站id，rent_type为租房方式（1整租2合租），默认页数为1页
    page_num = 1

    # 循环获取页面链接，并逐一获取内容，整理并入库
    con_list = get_content_links(city_short, rent_type, page_num, district_param, data_source)

    if not con_list:
        return False

    for xq_html, xq_url, list_tu_url, h_ajk_id in con_list:
        # print(xq_html)
        # print("*" * 200)
        con_handle(xq_html, city_code, rent_type, list_tu_url, h_ajk_id, xq_url, data_source)

    return True


if __name__ == '__main__':
    c = 'bj'
    c_code = '1101'
    p = "通州"
    crawler = '1'   # 爬虫所爬的网站id（1代表安居客）
    r_type = '1'    # 1整租2合租

    flag = ajk_zf_main(c, p, c_code, crawler, r_type)
